{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = 'all'  # default is ‘last_expr'\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/home/mink/notebooks/CameraTraps')  # append this repo to PYTHONPATH\n",
    "sys.path.append('/home/mink/lib/ai4eutils')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "from collections import Counter, defaultdict\n",
    "from random import sample\n",
    "from shutil import copyfile\n",
    "from multiprocessing.pool import ThreadPool\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "\n",
    "import path_utils  # ai4eutils\n",
    "\n",
    "from data_management.megadb.schema import sequences_schema_check\n",
    "from data_management.megadb.converters.cct_to_megadb import process_sequences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# uw_gardner, folder Processed Images, bird images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'uw_gardner'\n",
    "\n",
    "container_root = '/mink_disk_0/camtraps/uw-gardner/'\n",
    "# no path prefix - this dataset was added to the datasets table previously\n",
    "\n",
    "path_to_output = f'/home/mink/camtraps/data/megadb_jsons/{dataset_name}.json' \n",
    "path_to_output_temp = f'/home/mink/camtraps/data/megadb_jsons/{dataset_name}_temp.json' "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 0 - Add an entry to the `datasets` table\n",
    "\n",
    "Done"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1 - Prepare the `sequence` objects to insert into the database\n",
    "\n",
    "Species and location in a CSV (all bird entries, no human and almost no vehicle) but no sequence info.\n",
    "\n",
    "Only 3430 images are both in the CSV and in the \"Processed Images\" folder."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_path = os.path.join(container_root, 'Processed Images/Bassing_BirdImages_2021-01-25.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\n",
    "    csv_path,\n",
    "    usecols=['File', 'RelativePath', 'Folder', 'DateTime', 'CameraLocation', 'Count', 'Species']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39069"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>File</th>\n",
       "      <th>RelativePath</th>\n",
       "      <th>Folder</th>\n",
       "      <th>DateTime</th>\n",
       "      <th>CameraLocation</th>\n",
       "      <th>Species</th>\n",
       "      <th>Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>24413</th>\n",
       "      <td>RCNX6667.JPG</td>\n",
       "      <td>Camera_1\\C116_5.29.19\\DCIM\\100RECNX</td>\n",
       "      <td>NE6248</td>\n",
       "      <td>2018-11-12 10:47:05</td>\n",
       "      <td>NE6248_1</td>\n",
       "      <td>Turkey</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24823</th>\n",
       "      <td>RCNX8234.JPG</td>\n",
       "      <td>Camera_1\\C116_5.29.19\\DCIM\\100RECNX</td>\n",
       "      <td>NE6248</td>\n",
       "      <td>2019-05-05 08:58:09</td>\n",
       "      <td>NE6248_1</td>\n",
       "      <td>Turkey</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               File                         RelativePath  Folder  \\\n",
       "24413  RCNX6667.JPG  Camera_1\\C116_5.29.19\\DCIM\\100RECNX  NE6248   \n",
       "24823  RCNX8234.JPG  Camera_1\\C116_5.29.19\\DCIM\\100RECNX  NE6248   \n",
       "\n",
       "                  DateTime CameraLocation Species  Count  \n",
       "24413  2018-11-12 10:47:05       NE6248_1  Turkey      1  \n",
       "24823  2019-05-05 08:58:09       NE6248_1  Turkey      1  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)\n",
    "df.sample(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Grouse Spp', 'Common Raven', 'Turkey', 'Bird Spp', 'Raptor Spp'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Species'].unique()  # spp = multiple species"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "247797"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "247752"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "folder = os.path.join(container_root, 'Processed Images')\n",
    "\n",
    "paths = path_utils.recursive_file_list(folder)\n",
    "len(paths)\n",
    "paths = sorted([p.split(container_root)[1] for p in paths if path_utils.is_image_file(p) and not os.path.basename(p).startswith('.')])\n",
    "len(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Processed Images/NE1483_complete/Camera_32/C27_7.27.18/DCIM/100RECNX/RCNX0011.JPG'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paths[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "39069it [00:04, 8647.15it/s]\n"
     ]
    }
   ],
   "source": [
    "files_in_csv = []\n",
    "\n",
    "for i_row, row in tqdm(df.iterrows()):\n",
    "    fn = os.path.join(row['Folder'], row['RelativePath'].replace('\\\\', '/'), row['File'])\n",
    "    files_in_csv.append(fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "paths_set = set(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_set = set(files_in_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "intersection = paths_set.intersection(csv_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3430"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(intersection)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Processed Images/NE1483_complete/Camera_32/C80_10.4.18/DCIM/100RECNX/RCNX0611.JPG'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files_in_csv[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Processed Images/NE1483_complete/Camera_32/C27_7.27.18/DCIM/100RECNX/RCNX0001.JPG'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paths[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "39069it [00:04, 8750.13it/s]\n"
     ]
    }
   ],
   "source": [
    "locations = set()\n",
    "sequences = []\n",
    "\n",
    "for i_row, row in tqdm(df.iterrows()):\n",
    "    fn = os.path.join('Processed Images', row['Folder'], row['RelativePath'].replace('\\\\', '/'), row['File'])\n",
    "\n",
    "    if fn not in intersection:\n",
    "        continue\n",
    "    \n",
    "    clss = row['Species'].lower()\n",
    "    \n",
    "    seq_id = '_'.join([row['Folder'], row['RelativePath'].replace('\\\\', '_'), row['File'].split('.')[0]]).replace('.', '-')\n",
    "    assert '/' not in seq_id, row\n",
    "\n",
    "    location = row['CameraLocation']\n",
    "    locations.add(location)\n",
    "        \n",
    "    sequences.append({\n",
    "        'dataset': dataset_name,\n",
    "        'seq_id': f'dummy_{seq_id}',\n",
    "        'images': [\n",
    "            {\n",
    "                'file': fn,\n",
    "                'frame_num': 1, # only one image, but easier for ingesting the annotations\n",
    "                'datetime': row['DateTime'], # extra field,\n",
    "            }\n",
    "        ],\n",
    "        'location': location,\n",
    "        'class': [clss],\n",
    "        'count': row['Count']\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3430"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[{'dataset': 'uw_gardner',\n",
       "  'seq_id': 'dummy_NE4891_Camera_5_C4_9-11-18_DCIM_105RECNX_RCNX0342',\n",
       "  'images': [{'file': 'Processed Images/NE4891/Camera_5/C4_9.11.18/DCIM/105RECNX/RCNX0342.JPG',\n",
       "    'frame_num': 1,\n",
       "    'datetime': '2018-07-16 12:28:10'}],\n",
       "  'location': 'NE4891_5',\n",
       "  'class': ['turkey'],\n",
       "  'count': 1},\n",
       " {'dataset': 'uw_gardner',\n",
       "  'seq_id': 'dummy_NE5740_Camera_15_C36_7-27-18_DCIM_116RECNX_RCNX0457',\n",
       "  'images': [{'file': 'Processed Images/NE5740/Camera_15/C36_7.27.18/DCIM/116RECNX/RCNX0457.JPG',\n",
       "    'frame_num': 1,\n",
       "    'datetime': '2018-06-16 17:01:26'}],\n",
       "  'location': 'NE5740_15',\n",
       "  'class': ['common raven'],\n",
       "  'count': 2}]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sequences)\n",
    "sample(sequences, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(locations)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2 - Pass the schema check\n",
    "\n",
    "Once your metadata are in the MegaDB format for `sequence` items, we check that they conform to the format's schema.\n",
    "\n",
    "If the format conforms, the following messages will be printed:\n",
    "\n",
    "```\n",
    "Verified that the sequence items meet requirements not captured by the schema.\n",
    "Verified that the sequence items conform to the schema.\n",
    "```\n",
    "\n",
    "For large datasets, the second step will take some time (~ a minute). \n",
    "\n",
    "Otherwise there will be an error message describing what's wrong. Please fix the issues until all checks are passed. You might need to write some snippets of code to loop through the `sequence` items to understand which entries have problems."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Verified that the sequence items meet requirements not captured by the schema.\n",
      "Verified that the sequence items conform to the schema.\n",
      "CPU times: user 469 ms, sys: 0 ns, total: 469 ms\n",
      "Wall time: 467 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "sequences_schema_check.sequences_schema_check(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(path_to_output_temp, 'w', encoding='utf-8') as f:\n",
    "    json.dump(sequences, f, indent=1, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 2b - copy non-empties to flat folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def copy_file(src_path, dst_path):\n",
    "    return copyfile(src_path, dst_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3430/3430 [00:00<00:00, 114156.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 22.3 ms, sys: 12 ms, total: 34.3 ms\n",
      "Wall time: 32.6 ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "path_pairs = []\n",
    "for seq in tqdm(sequences):\n",
    "    \n",
    "    if seq['class'][0] == 'empty':\n",
    "        continue\n",
    "    \n",
    "    seq_id = seq['seq_id']\n",
    "    \n",
    "    im = seq['images'][0]\n",
    "    frame = im['frame_num']\n",
    "    \n",
    "    src_path = os.path.join(container_root, im['file'])\n",
    "    assert os.path.exists(src_path), src_path\n",
    "\n",
    "    dst_path = os.path.join('/mink_disk_0/camtraps/imerit12d', \n",
    "                            f'{dataset_name}.seq{seq_id}.frame{frame}.jpg')\n",
    "    path_pairs.append((src_path, dst_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3430"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "('/mink_disk_0/camtraps/uw-gardner/Processed Images/NE7302/Camera_84/C69_2.26.19/DCIM/102RECNX/RCNX1972.JPG',\n",
       " '/mink_disk_0/camtraps/imerit12d/uw_gardner.seqdummy_NE7302_Camera_84_C69_2-26-19_DCIM_102RECNX_RCNX1972.frame1.jpg')"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(path_pairs)  # non-empty images out of total of 19221 (38%)\n",
    "path_pairs[3000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.63 s, sys: 5.46 s, total: 8.09 s\n",
      "Wall time: 13.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "with ThreadPool(8) as pool:\n",
    "    dst_paths = pool.starmap(copy_file, path_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3430"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dst_paths)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:cameratraps] *",
   "language": "python",
   "name": "conda-env-cameratraps-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
